#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/8/15 18:34
# @Author  : youfeng

# 更新调度
import importlib
import os
import time
from imp import load_source
from multiprocessing.pool import ThreadPool

from common.util import Util
from config import LOCK_REDIS_CONF, TEMP_REDIS_CONF
from dao.proxy_dao import ProxyDAO
from dao.proxy_lock_dao import ProxyLockDAO
from enum.proxy_key import ProxyKey
from ext.statistics_service import StatisticsService


class UpdateService(object):
    __TAG__ = 'all'

    __PROXY_STORE_KEY = "UpdateService:store:"

    # 最小生存时间 秒
    __PROXY_MIN_TTL = 30

    def __init__(self, base_path, task_prefix="proxys", log=None):
        self.log = log
        self.__task_prefix = task_prefix
        self.__base_path = base_path
        self.__proxy_lock_dao = ProxyLockDAO(LOCK_REDIS_CONF, self.log)
        self.__proxy_dao = ProxyDAO(TEMP_REDIS_CONF, self.log)
        self.__statistics_service = StatisticsService(self.__proxy_dao, self.log)
        self.pool = ThreadPool(processes=32)

    def __get_proxy_list(self, module, code_file, code_path):
        self.log.info("开始获取代理: class = {}".format(module.__CLASS__))
        try:
            class_module = load_source(code_file.split(".")[0], code_path)
            if module.__CLASS__ in dir(class_module):
                # 初始化类的实例并存入dict中，
                clazz = getattr(class_module, module.__CLASS__)
                clazz_instance = clazz(self.log)
                proxy_list = clazz_instance.get_proxy_list()
                if not isinstance(proxy_list, list):
                    self.log.error("返回值代理列表类型异常: {}".format(type(proxy_list)))
                    return [], None

                self.log.info("代理获取完成: class = {}".format(module.__CLASS__))
                return proxy_list, clazz_instance.get_proxy_tag()
        except Exception as e:
            self.log.error("获取代理失败: ")
            self.log.exception(e)

        return [], None

    # 对代理进行排序
    def __sorted_proxy_list(self, total_proxy_list):

        temp_proxy_list = []
        for proxy in total_proxy_list:
            if "@" in proxy:
                ip = proxy.split("@")[1]
            else:
                ip = proxy.split("//")[1]

            temp_proxy_list.append((ip, proxy))

        temp_sorted_list = sorted(temp_proxy_list, key=lambda x: x[0])
        result_list = [item[1] for item in temp_sorted_list]

        return result_list

    def __get_proxy_from_server(self):
        result_list = []
        proxy_list_dict = {}

        tasks_dir = os.path.join(self.__base_path, self.__task_prefix)
        for parent_file in os.listdir(tasks_dir):
            parent_task_path = os.path.join(tasks_dir, parent_file)
            if not os.path.isdir(parent_task_path):
                continue

            for code_file in os.listdir(parent_task_path):
                code_path = os.path.join(parent_task_path, code_file)
                # 如果不是代码文件 则直接跳过
                if not os.path.isfile(code_path):
                    continue

                # 如果不是py文件 则直接跳过
                if not code_path.endswith(".py"):
                    continue

                # 先import文件 获得 __CLASS__ 与 __USED__变量
                import_path = ".".join((self.__task_prefix, parent_file, code_file.split(".")[0]))
                self.log.info("当前需要被导入的路径: import_path = {}".format(import_path))
                module = importlib.import_module(import_path)
                reload(module)
                if not hasattr(module, '__CLASS__'):
                    self.log.warn("当前文件没有包含__CLASS__变量: code_file = {}".format(code_file))
                    continue

                if not hasattr(module, '__USED__'):
                    self.log.warn("当前文件没有包含__USED__变量: code_file = {}".format(code_file))
                    continue

                if not module.__USED__:
                    self.log.info("当前代理未打开: proxy = {}".format(module.__CLASS__))
                    continue

                self.log.info("当前代理开关状态: __CLASS__ = {} __USED__ = {}".format(
                    module.__CLASS__, module.__USED__))

                result_list.append(self.pool.apply_async(self.__get_proxy_list, (module, code_file, code_path,)))

        self.log.info("所有线程加载完成，等待线程执行完成...")
        for result in result_list:
            proxy_list, tag = result.get()
            if tag is None:
                continue

            if tag not in proxy_list_dict:
                proxy_list_dict[tag] = proxy_list
            else:
                self.log.error("存在重复的标签: tag = {}".format(tag))

        return proxy_list_dict

    # 获得代理redis主键
    def __get_proxy_list_key(self, tag):
        return "{}{}".format(self.__PROXY_STORE_KEY, tag)

    # 从redis中读取上一次存储的代理
    def __get_proxy_from_redis(self, tags):
        proxy_list_dict = {}
        start_time = time.time()

        for tag in tags:
            redis_key = self.__get_proxy_list_key(tag)
            proxy_list = self.__proxy_dao.get_proxy_list_from_redis(redis_key)
            proxy_list_dict[tag] = proxy_list

        self.log.info("获取临时存储代理耗时: {} s".format(round(time.time() - start_time, 2)))
        return proxy_list_dict

    # 删除临时存储代理
    def __delete_temp_proxy_list(self, tags):
        start_time = time.time()
        for tag in tags:
            redis_key = self.__get_proxy_list_key(tag)
            self.__proxy_dao.delete_redis_proxy_list(redis_key)
        self.log.info("删除临时存储代理耗时: {} s".format(round(time.time() - start_time, 2)))

    # 存储最新代理
    def __set_temp_proxy_list_to_redis(self, merge_proxy_list_dict):
        start_time = time.time()
        for tag, proxy_list in merge_proxy_list_dict.iteritems():
            redis_key = self.__get_proxy_list_key(tag)
            self.__proxy_dao.set_proxy_list_to_redis(redis_key, proxy_list)
            self.log.info("存储: {} length = {}".format(tag, len(proxy_list)))
        self.log.info("存储临时存储代理耗时: {} s".format(round(time.time() - start_time, 2)))

    # 代理列表合并
    def __merge_proxy_list(self, redis_proxy_list_dict, server_proxy_list_dict):
        merge_proxy_list_dict = {}
        current_time = int(time.time())
        for tag, redis_proxy_list in redis_proxy_list_dict.iteritems():
            server_proxy_list = server_proxy_list_dict.get(tag)
            if not isinstance(server_proxy_list, list):
                continue

            ip_set = set()
            merge_list = []

            def merge(temp_list):
                for proxy_dict in temp_list:
                    proxy = proxy_dict.get(ProxyKey.PROXY_KEY)
                    if proxy is None:
                        continue

                    expire_time = proxy_dict.get(ProxyKey.EXPIRE_TIME_KEY)
                    if expire_time is None:
                        continue

                    # 对于时间快过期的代理进行剔除
                    if expire_time - current_time < self.__PROXY_MIN_TTL:
                        continue

                    ip = Util.get_ip_port_from_proxy(proxy)
                    if ip in ip_set:
                        continue

                    ip_set.add(ip)
                    merge_list.append(proxy_dict)

            # 合并代理
            # 先合并存量存在redis中的代理
            merge(redis_proxy_list)
            # 然后再合并从外网获取到的最新代理
            merge(server_proxy_list)

            merge_proxy_list_dict[tag] = merge_list
        return merge_proxy_list_dict

    # 提取应用层使用的代理信息
    def __get_app_used_proxy(self, merge_proxy_list_dict):

        proxy_list_dict = {}
        total_proxy_list = []
        for tag, proxy_list in merge_proxy_list_dict.iteritems():
            temp_list = self.__sorted_proxy_list([item.get(ProxyKey.PROXY_KEY) for item in proxy_list])
            total_proxy_list.extend(temp_list)
            proxy_list_dict[tag] = temp_list
        proxy_list_dict[self.__TAG__] = self.__sorted_proxy_list(total_proxy_list)
        return proxy_list_dict

    # 存储所有代理信息到缓存，用于统计监控
    def __save_statistics_proxy(self, merge_proxy_list_dict):
        for tag, proxy_list in merge_proxy_list_dict.iteritems():
            # 记录每天切面 切面只保存 14天
            self.__statistics_service.save_daily_proxy(tag, proxy_list)
            # 记录所有代理
            self.__statistics_service.save_total_proxy(tag, proxy_list)

    # 获得所有代理
    def __get_proxy_list_dict(self):

        start_time = time.time()

        # 从云上获取代理
        server_proxy_list_dict = self.__get_proxy_from_server()

        # 从redis中获取代理
        redis_proxy_list_dict = self.__get_proxy_from_redis(tags=server_proxy_list_dict.keys())

        # 合并最终可用代理
        merge_proxy_list_dict = self.__merge_proxy_list(redis_proxy_list_dict, server_proxy_list_dict)

        # 删除临时存储的代理信息
        self.__delete_temp_proxy_list(tags=server_proxy_list_dict.keys())

        # 存储最新合并的代理结果
        self.__set_temp_proxy_list_to_redis(merge_proxy_list_dict)

        # 得到最终返回给用户层的代理信息
        proxy_list_dict = self.__get_app_used_proxy(merge_proxy_list_dict)

        # 存储统计数据
        self.__save_statistics_proxy(merge_proxy_list_dict)

        self.log.info("获取所有代理耗时: {} s".format(round(time.time() - start_time, 2)))
        return proxy_list_dict

    # 启动更新调度
    def start(self):
        # 获取代理
        proxy_list_dict = self.__get_proxy_list_dict()

        if self.__proxy_lock_dao.set_proxy(proxy_list_dict):
            for tag, proxy_list in proxy_list_dict.iteritems():
                self.log.info("代理更新成功，更新数目: tag = {} proxy num = {}".format(tag, len(proxy_list_dict.get(tag))))
